#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
JD商城商品信息采集器 - 清理版
专注于全量商品数据采集，为Redis监听做准备
"""

import time
import re
import json
import random
import logging
import os
import sys
from datetime import datetime
from typing import List, Dict, Any, Optional
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import TimeoutException, NoSuchElementException, ElementClickInterceptedException

# 添加项目根目录到路径
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if project_root not in sys.path:
    sys.path.insert(0, project_root)

# 导入WebDriver管理器
try:
    from webdriver_manager_util import WebDriverManager
except ImportError:
    print("⚠️ 无法导入webdriver_manager_util，将使用传统方式")
    WebDriverManager = None

from JD.config import JDConfig


class JDScraper:
    """JD商城采集器 - 清理版"""
    
    def __init__(self, headless: bool = False, debug: bool = False):
        """初始化JD采集器"""
        self.debug = debug
        self.headless = headless
        self.driver = None
        self.wait = None
        
        # 随机选择User-Agent
        self.user_agent = random.choice(JDConfig.USER_AGENTS)
        
        # GUI回调函数
        self.login_confirmation_callback = None
        self.verification_callback = None
        self.credential_input_callback = None
        
        # 配置日志
        self._setup_logging()
        self.logger.info("JDScraper 初始化...")
        self.logger.info(f"选择User-Agent: {self.user_agent[:80]}...")
        
        # 初始化浏览器
        self._init_driver()
    
    def set_login_confirmation_callback(self, callback):
        """设置登录确认回调函数"""
        self.login_confirmation_callback = callback
    
    def set_verification_callback(self, callback):
        """设置验证码处理回调函数"""
        self.verification_callback = callback
    
    def set_credential_input_callback(self, callback):
        """设置凭证输入回调函数"""
        self.credential_input_callback = callback
        
    def _setup_logging(self):
        """配置日志系统"""
        log_level = logging.DEBUG if self.debug else logging.INFO
        
        # 创建logger
        self.logger = logging.getLogger('JDScraper')
        self.logger.setLevel(log_level)
        
        # 避免重复添加handler
        if not self.logger.handlers:
            # 文件handler
            try:
                file_handler = logging.FileHandler(JDConfig.LOG_FILE, encoding='utf-8')
                file_handler.setLevel(log_level)
                
                # 控制台handler  
                console_handler = logging.StreamHandler()
                console_handler.setLevel(log_level)
                
                # 格式化
                formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
                file_handler.setFormatter(formatter)
                console_handler.setFormatter(formatter)
                
                self.logger.addHandler(file_handler)
                self.logger.addHandler(console_handler)
                
            except Exception as e:
                print(f"日志配置失败: {e}")
                # 如果文件handler失败，至少配置控制台handler
                console_handler = logging.StreamHandler()
                console_handler.setLevel(log_level)
                formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
                console_handler.setFormatter(formatter)
                self.logger.addHandler(console_handler)

    def _init_driver(self):
        """初始化Chrome WebDriver"""
        try:
            chrome_options = Options()
            
            # 基础配置
            if self.headless:
                chrome_options.add_argument('--headless')
            
            # 应用配置中的Chrome选项
            for option in JDConfig.CHROME_OPTIONS:
                chrome_options.add_argument(option)
            
            # 反检测配置
            chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
            chrome_options.add_experimental_option('useAutomationExtension', False)
            
            # 使用预选的随机用户代理
            chrome_options.add_argument(f'--user-agent={self.user_agent}')
            self.logger.info(f"应用用户代理: {self.user_agent[:50]}...")
            
            # 尝试使用WebDriverManager自动管理ChromeDriver
            if WebDriverManager:
                try:
                    self.logger.info("🔍 使用WebDriverManager自动配置ChromeDriver...")
                    manager = WebDriverManager(logger=self.logger)
                    self.driver = manager.get_chrome_driver(options=chrome_options, headless=self.headless)
                    self.logger.info("✅ ChromeDriver自动配置成功")
                except Exception as e:
                    self.logger.warning(f"⚠️ WebDriverManager初始化失败: {e}")
                    self.logger.info("🔄 回退到传统方式...")
                    self._init_driver_legacy(chrome_options)
            else:
                # 如果没有WebDriverManager，使用传统方式
                self._init_driver_legacy(chrome_options)
            
            self.wait = WebDriverWait(self.driver, JDConfig.DEFAULT_TIMEOUT)
            
            # 执行反检测脚本
            self._execute_anti_detection_script()
            
            # 设置窗口大小为常见分辨率
            window_sizes = [(1920, 1080), (1366, 768), (1536, 864), (1440, 900)]
            width, height = random.choice(window_sizes)
            self.driver.set_window_size(width, height)
            self.logger.info(f"设置窗口大小: {width}x{height}")
            
            self.logger.info("Chrome WebDriver 初始化成功")
            
        except Exception as e:
            self.logger.error(f"Chrome WebDriver 初始化失败: {e}")
            raise
    
    def _init_driver_legacy(self, chrome_options):
        """传统方式初始化driver（备用方案）"""
        try:
            # 尝试使用本地chromedriver
            current_dir = os.path.dirname(os.path.abspath(__file__))
            chromedriver_path = os.path.join(current_dir, 'chromedriver.exe')
            
            if os.path.exists(chromedriver_path):
                self.logger.info(f"📂 使用本地ChromeDriver: {chromedriver_path}")
                service = Service(chromedriver_path)
                self.driver = webdriver.Chrome(service=service, options=chrome_options)
            else:
                self.logger.info("🔄 使用系统PATH中的ChromeDriver")
                self.driver = webdriver.Chrome(options=chrome_options)
                
        except Exception as e:
            self.logger.error(f"❌ 传统方式初始化ChromeDriver失败: {e}")
            raise
    
    def _execute_anti_detection_script(self):
        """执行反检测脚本"""
        script = """
        Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
        Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
        Object.defineProperty(navigator, 'languages', {get: () => ['zh-CN', 'zh', 'en']});
        window.chrome = {runtime: {}};
        Object.defineProperty(navigator, 'permissions', {get: () => ({query: () => Promise.resolve({state: 'granted'})})});
        """
        try:
            self.driver.execute_script(script)
            self.logger.info("反检测脚本执行完成")
        except Exception as e:
            self.logger.warning(f"反检测脚本执行失败: {e}")
    
    def _human_delay(self, min_delay: float = 1.0, max_delay: float = 3.0, login_mode: bool = False):
        """模拟人类操作延时"""
        if login_mode:
            # 登录模式使用更长的延时
            delay = random.uniform(2.0, 4.0)
        else:
            delay = random.uniform(min_delay, max_delay)
        time.sleep(delay)
    
    def _simulate_human_behavior(self, element):
        """模拟人类的鼠标行为"""
        try:
            actions = ActionChains(self.driver)
            
            # 随机选择行为模式
            behavior_type = random.choice(['move_and_click', 'scroll_then_click', 'hover_then_click'])
            
            if behavior_type == 'move_and_click':
                # 先移动到元素附近，然后移动到元素，最后点击
                offset_x = random.randint(-50, 50)
                offset_y = random.randint(-30, 30)
                actions.move_to_element_with_offset(element, offset_x, offset_y)
                actions.pause(random.uniform(0.3, 0.8))
                actions.move_to_element(element)
                actions.pause(random.uniform(0.2, 0.5))
                actions.click()
                
            elif behavior_type == 'scroll_then_click':
                # 滚动到元素可见，然后点击
                self.driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", element)
                self._human_delay(0.5, 1.2)
                actions.move_to_element(element)
                actions.pause(random.uniform(0.3, 0.6))
                actions.click()
                
            elif behavior_type == 'hover_then_click':
                # 悬停一会儿再点击
                actions.move_to_element(element)
                actions.pause(random.uniform(0.5, 1.0))
                actions.click()
            
            actions.perform()
            return True
            
        except Exception as e:
            self.logger.debug(f"人性化点击失败: {e}")
            return False
    
    def _page_interaction_warmup(self):
        """页面交互预热 - 模拟真实用户的浏览行为"""
        try:
            # 随机滚动页面
            scroll_actions = random.randint(2, 5)
            for _ in range(scroll_actions):
                # 随机滚动方向和距离
                scroll_distance = random.randint(100, 500)
                direction = random.choice([1, -1])
                self.driver.execute_script(f"window.scrollBy(0, {scroll_distance * direction});")
                self._human_delay(0.3, 0.8)
            
            # 随机移动鼠标
            if random.choice([True, False]):
                try:
                    body = self.driver.find_element(By.TAG_NAME, "body")
                    actions = ActionChains(self.driver)
                    for _ in range(random.randint(1, 3)):
                        offset_x = random.randint(0, 500)
                        offset_y = random.randint(0, 300)
                        actions.move_to_element_with_offset(body, offset_x, offset_y)
                        actions.pause(random.uniform(0.2, 0.5))
                    actions.perform()
                except:
                    pass
                    
        except Exception as e:
            self.logger.debug(f"页面预热失败: {e}")
    
    def _random_product_browse_delay(self):
        """商品间浏览的随机延迟"""
        # 模拟真实用户在商品间的浏览时间
        delay_patterns = [
            (3, 8),    # 短时间浏览
            (8, 15),   # 中等浏览时间
            (15, 25),  # 长时间浏览（偶尔）
        ]
        
        # 90%概率短浏览，8%中等，2%长时间
        pattern = random.choices(delay_patterns, weights=[90, 8, 2])[0]
        delay = random.uniform(*pattern)
        
        print(f"⏱️  模拟用户浏览商品 ({delay:.1f}秒)...")
        time.sleep(delay)
    
    def _check_for_verification_issues(self) -> bool:
        """检查是否出现验证码或其他需要人工处理的问题"""
        try:
            # 检查常见的验证码元素
            verification_selectors = [
                # 滑块验证码
                '.JDJRV-slide-inner',
                '.JDJRV-bigimg', 
                '.slider_bg',
                # 点击验证码
                '.JDJRV-img-panel',
                '.verify-image',
                # 登录提示
                '.login-form',
                '#loginname',
                # 其他验证元素
                '.captcha',
                '.verification',
                '.verify',
                '[id*="verify"]',
                '[class*="captcha"]',
                '[class*="verify"]'
            ]
            
            for selector in verification_selectors:
                elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                if elements and any(elem.is_displayed() for elem in elements):
                    self.logger.warning(f"检测到验证元素: {selector}")
                    return True
                    
            # 检查页面标题是否包含验证相关关键词
            title = self.driver.title.lower()
            verification_keywords = ['验证', 'verify', 'captcha', '滑动', '点击', '登录']
            if any(keyword in title for keyword in verification_keywords):
                self.logger.warning(f"页面标题包含验证关键词: {title}")
                return True
                
            # 检查URL是否重定向到验证页面
            current_url = self.driver.current_url.lower()
            verification_url_keywords = [
                'verify', 'captcha', 'login', 'passport', 
                'risk_handler', 'privatedomain',  # JD特有的验证页面
                'cfe.m.jd.com', 'plogin.m.jd.com'  # JD验证相关域名
            ]
            
            if any(keyword in current_url for keyword in verification_url_keywords):
                if 'item.jd.com' not in current_url:  # 不是商品页面
                    self.logger.warning(f"URL重定向到验证页面: {current_url}")
                    return True
            
            # 检查页面内容是否包含验证相关文本
            try:
                page_text = self.driver.find_element(By.TAG_NAME, "body").text.lower()
                verification_text_keywords = [
                    '滑动验证', '点击验证', '图形验证码', '请完成验证', 
                    '安全验证', '人机验证', '验证码', '请稍候'
                ]
                if any(keyword in page_text for keyword in verification_text_keywords):
                    self.logger.warning(f"页面内容包含验证关键词")
                    return True
            except:
                pass
                    
            return False
            
        except Exception as e:
            self.logger.error(f"验证检查失败: {e}")
            return False
    
    def _handle_verification_pause(self, product: Dict[str, Any]) -> bool:
        """处理验证码暂停，等待人工干预
        
        Returns:
            bool: True表示应该重新获取该商品详情，False表示跳过该商品
        """
        try:
            product_title = product.get('title', '未知商品')[:50]
            current_url = self.driver.current_url
            
            print("\n" + "="*80)
            print("🚨 检测到验证码或页面异常！")
            print("="*80)
            print(f"📦 当前商品: {product_title}...")
            print(f"🌐 当前URL: {current_url}")
            print(f"📄 页面标题: {self.driver.title}")
            print("-"*80)
            print("📋 请在浏览器中完成以下操作:")
            print("   1. 完成滑块验证（如果有）")
            print("   2. 完成点击验证码（如果有）")
            print("   3. 输入图形验证码（如果有）")
            print("   4. 完成登录验证（如果需要）")
            print("   5. 验证完成后，程序会自动关闭当前页面")
            print("   6. 然后从商品列表页重新获取该商品详情")
            print("-"*80)
            print("💡 新的处理流程:")
            print("   - 选择'c'继续：验证完成后会重新获取该商品详情")
            print("   - 选择's'跳过：跳过该商品，继续处理下一个")
            print("   - 选择'q'退出：完全退出程序")
            print("-"*80)
            
            # 使用GUI回调或命令行交互
            if hasattr(self, 'verification_callback') and self.verification_callback:
                # 使用GUI回调
                action = self.verification_callback("京东", product_title)
                if action == 'continue':
                    print("✅ 验证处理完成，将关闭当前页面并重新获取该商品详情")
                    return True
                elif action == 'skip':
                    print("⏸️ 跳过该商品，继续处理下一个")
                    return False
                elif action == 'quit':
                    print("🚪 退出程序")
                    exit(0)
                else:
                    print("❌ 用户取消操作，跳过该商品")
                    return False
            else:
                # 命令行交互
                while True:
                    try:
                        choice = input("✋ 处理完成后输入: 'c'继续重新获取 / 's'跳过该商品 / 'q'退出程序: ").strip().lower()
                        
                        if choice == 'c':
                            print("✅ 验证处理完成，将关闭当前页面并重新获取该商品详情")
                            return True  # 返回True表示需要重新获取详情
                            
                        elif choice == 's':
                            print("⏭️  跳过当前商品，继续下一个")
                            return False  # 返回False表示跳过该商品
                            
                        elif choice == 'q':
                            print("🛑 用户选择退出程序")
                            raise KeyboardInterrupt("用户手动退出")
                            
                        else:
                            print("❌ 无效输入，请输入 'c'、's' 或 'q'")
                            continue
                            
                    except EOFError:
                        print("\n🛑 检测到输入中断，跳过当前商品")
                        return False
                    except KeyboardInterrupt:
                        print("\n🛑 用户中断程序")
                        raise
                    
        except Exception as e:
            self.logger.error(f"验证暂停处理失败: {e}")
            print("❌ 处理验证暂停时出错，跳过当前商品")
            return False
    
    def _close_current_detail_and_return(self):
        """关闭当前详情页面并返回主列表页面"""
        try:
            current_handles = self.driver.window_handles
            if len(current_handles) > 1:
                # 关闭当前窗口
                self.driver.close()
                # 切换到主窗口（通常是第一个）
                self.driver.switch_to.window(current_handles[0])
                self.logger.info("已关闭详情页面，返回主列表页面")
            else:
                # 如果只有一个窗口，尝试返回上一页
                self.driver.back()
                self._human_delay(2, 3)
                self.logger.info("已返回上一页")
        except Exception as e:
            self.logger.error(f"关闭详情页面失败: {e}")
    
    def _ensure_back_to_product_list(self):
        """确保回到商品列表页面"""
        try:
            # 🔧 修复：添加窗口状态检查和恢复
            self._validate_and_fix_window_state()
            
            current_url = self.driver.current_url
            
            # 检查是否已经在商品列表页
            if 'mall.jd.com' in current_url and ('index' in current_url or 'view_search' in current_url):
                self.logger.info("✅ 已在商品列表页")
                return True
            
            # 尝试关闭多余的窗口/标签页
            current_handles = self.driver.window_handles
            if len(current_handles) > 1:
                # 关闭除第一个窗口外的所有窗口
                for handle in current_handles[1:]:
                    try:
                        self.driver.switch_to.window(handle)
                        self.driver.close()
                    except:
                        pass
                
                # 切换回第一个窗口
                self.driver.switch_to.window(current_handles[0])
                self.logger.info("✅ 已关闭多余窗口并回到主窗口")
            
            # 检查主窗口是否是商品列表页
            current_url = self.driver.current_url
            if 'mall.jd.com' not in current_url:
                self.logger.warning(f"当前页面不是商品列表页: {current_url}")
                # 可以考虑重新导航到商品列表页，但这里先记录警告
                return False
            
            return True
            
        except Exception as e:
            self.logger.error(f"确保回到商品列表页失败: {e}")
            return False
    
    def _validate_and_fix_window_state(self):
        """验证并修复窗口状态"""
        try:
            self.logger.info("🔍 检查窗口状态...")
            
            # 检查driver是否还有效
            try:
                current_handles = self.driver.window_handles
                self.logger.info(f"📊 当前窗口数量: {len(current_handles)}")
            except Exception as e:
                self.logger.error(f"❌ Driver无效或窗口已关闭: {e}")
                raise Exception("Driver状态异常，需要重新初始化")
            
            # 如果没有窗口，说明所有窗口都被关闭了
            if not current_handles:
                self.logger.error("❌ 所有窗口都被关闭")
                raise Exception("所有窗口都被关闭，需要重新初始化")
            
            # 确保在有效窗口中
            try:
                current_url = self.driver.current_url
                self.logger.info(f"📍 当前页面: {current_url}")
            except Exception as e:
                self.logger.warning(f"⚠️ 当前窗口无效，尝试切换到第一个窗口: {e}")
                # 切换到第一个有效窗口
                for handle in current_handles:
                    try:
                        self.driver.switch_to.window(handle)
                        current_url = self.driver.current_url
                        self.logger.info(f"✅ 成功切换到窗口: {current_url}")
                        break
                    except:
                        continue
                else:
                    raise Exception("无法找到有效窗口")
            
            # 如果当前页面不是商品列表页，尝试导航回去
            current_url = self.driver.current_url
            if 'mall.jd.com' not in current_url or ('index' not in current_url and 'view_search' not in current_url):
                self.logger.warning(f"⚠️ 当前不在商品列表页: {current_url}")
                # 可以尝试使用浏览器的后退功能或重新导航
                try:
                    self.driver.back()
                    self._human_delay(2, 3)
                    new_url = self.driver.current_url
                    if 'mall.jd.com' in new_url:
                        self.logger.info(f"✅ 通过后退返回商品列表页: {new_url}")
                    else:
                        self.logger.warning(f"⚠️ 后退后仍不在商品列表页: {new_url}")
                except Exception as e:
                    self.logger.warning(f"⚠️ 后退操作失败: {e}")
            
            self.logger.info("✅ 窗口状态检查完成")
            
        except Exception as e:
            self.logger.error(f"❌ 窗口状态验证失败: {e}")
            # 这里可以考虑重新初始化driver或抛出异常让上层处理
            raise
    
    def login_jd(self) -> bool:
        """JD登录"""
        try:
            self.logger.info("开始JD登录流程...")
            
            # 访问登录页面
            self.driver.get(JDConfig.LOGIN_URL)
            self._human_delay(2, 4, login_mode=True)
            
            # 等待登录页面加载
            self.wait.until(EC.presence_of_element_located((By.ID, "loginname")))
            self.logger.info("登录页面加载完成")
            
            # 获取用户名密码（支持GUI回调）
            if hasattr(self, 'credential_input_callback') and self.credential_input_callback:
                # 使用GUI回调获取凭证
                credentials = self.credential_input_callback("京东")
                if not credentials:
                    self.logger.error("用户取消登录")
                    return False
                username, password = credentials
            else:
                # 命令行交互式输入
                print("\n" + "="*60)
                print("🔐 JD商城登录")
                print("="*60)
                
                username = input("请输入JD用户名/手机号/邮箱: ").strip()
                password = input("请输入JD密码: ").strip()
            
            if not username or not password:
                self.logger.error("用户名或密码不能为空")
                return False
            
            # 输入用户名
            username_input = self.driver.find_element(By.ID, "loginname")
            username_input.clear()
            for char in username:
                username_input.send_keys(char)
                time.sleep(random.uniform(0.05, 0.15))
            
            self.logger.info("已输入用户名")
            self._human_delay(1, 2, login_mode=True)
            
            # 输入密码
            password_input = self.driver.find_element(By.ID, "nloginpwd")
            password_input.clear()
            for char in password:
                password_input.send_keys(char)
                time.sleep(random.uniform(0.05, 0.15))
            
            self.logger.info("已输入密码")
            self._human_delay(1, 2, login_mode=True)
            
            # 点击登录按钮
            login_btn = self.driver.find_element(By.ID, "loginsubmit")
            self.driver.execute_script("arguments[0].click();", login_btn)
            self.logger.info("已点击登录按钮")
            
            print("✅ 已点击登录按钮，等待页面响应...")
            
            # 检测验证码或登录结果
            self._human_delay(3, 5, login_mode=True)
            
            # 等待并处理可能的验证
            max_wait_time = 60  # 最多等待60秒
            start_time = time.time()
            
            while time.time() - start_time < max_wait_time:
                current_url = self.driver.current_url
                
                # 检查是否需要滑块验证
                try:
                    slider = self.driver.find_element(By.CLASS_NAME, "JDJRV-slide-inner")
                    if slider.is_displayed():
                        # 使用GUI回调或命令行交互
                        if hasattr(self, 'login_confirmation_callback') and self.login_confirmation_callback:
                            # 使用GUI回调
                            confirmed = self.login_confirmation_callback("京东")
                            if not confirmed:
                                self.logger.error("用户取消滑块验证")
                                return False
                        else:
                            # 命令行交互
                            print("🔍 检测到滑块验证，请手动完成验证后按回车继续...")
                            input("验证完成后按回车: ")
                        time.sleep(2)
                        continue
                except:
                    pass
                
                # 检查是否需要短信验证
                try:
                    sms_input = self.driver.find_element(By.ID, "authcode")
                    if sms_input.is_displayed():
                        # 获取短信验证码（支持GUI回调）
                        if hasattr(self, 'credential_input_callback') and self.credential_input_callback:
                            # 使用GUI回调获取验证码（传递特殊标识）
                            result = self.credential_input_callback("京东", sms_verification=True)
                            if not result:
                                self.logger.error("用户取消短信验证")
                                return False
                            sms_code = result[0] if isinstance(result, tuple) else result
                        else:
                            # 命令行交互
                            print("📱 检测到短信验证，请查收短信并输入验证码...")
                            sms_code = input("请输入短信验证码: ").strip()
                        if sms_code:
                            sms_input.clear()
                            sms_input.send_keys(sms_code)
                            
                            # 点击验证按钮
                            verify_btn = self.driver.find_element(By.ID, "smsbtn")
                            verify_btn.click()
                            time.sleep(3)
                        continue
                except:
                    pass
                
                # 检查是否登录成功
                if "passport.jd.com" not in current_url and "login" not in current_url:
                    break
                    
                time.sleep(2)
            
            # 验证登录状态
            cookies = self.driver.get_cookies()
            self.logger.info(f"提取了 {len(cookies)} 个cookies")
            
            if len(cookies) > 10:  # 一般登录后会有很多cookies
                self.logger.info("JD登录成功！")
                return True
            else:
                self.logger.error("JD登录失败")
                return False
                
        except Exception as e:
            self.logger.error(f"JD登录过程中出错: {e}")
            return False

    def navigate_to_shop(self, shop_url: str) -> bool:
        """导航到店铺页面"""
        try:
            self.logger.info(f"正在导航到店铺页面: {shop_url}")
            
            self.driver.get(shop_url)
            self._human_delay(3, 5)
            
            # 等待页面加载
            self.wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            
            # 提取店铺信息
            shop_info = self._extract_shop_info(shop_url)
            self.logger.info(f"提取店铺信息: {shop_info}")
            
            self.logger.info("成功导航到店铺页面")
            return True
            
        except Exception as e:
            self.logger.error(f"导航到店铺失败: {e}")
            return False
    
    def _extract_shop_info(self, shop_url: str) -> Dict[str, Any]:
        """提取店铺信息"""
        shop_info = {
            'shop_name': '',
            'shop_id': '',
            'shop_url': shop_url,
            'extracted_at': datetime.now().isoformat()
        }
        
        try:
            # 从URL提取店铺ID
            match = re.search(r'view_search-(\d+)', shop_url)
            if match:
                shop_info['shop_id'] = match.group(1)
            
            # 尝试提取店铺名称
            try:
                shop_name_elem = self.driver.find_element(By.CSS_SELECTOR, ".shop-name, .store-name, .shop-title")
                shop_info['shop_name'] = shop_name_elem.text.strip()
            except:
                pass
                
        except Exception as e:
            self.logger.warning(f"提取店铺信息失败: {e}")
        
        return shop_info

    def get_product_list(self, max_pages: int = None, max_products_per_page: int = None) -> List[Dict[str, Any]]:
        """获取商品列表 - 支持智能分页"""
        products = []
        
        try:
            # 等待页面加载
            self.wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            self._human_delay(2, 4)
            
            # 获取分页信息
            pagination_info = self._get_pagination_info()
            current_page = pagination_info['current_page']
            total_pages = pagination_info['total_pages']
            total_products = pagination_info['total_products']
            
            print(f"\n📊 JD分页信息:")
            print(f"   📄 当前页: {current_page}/{total_pages}")
            print(f"   📦 总商品数: {total_products}")
            print(f"   🎯 预计每页约20个商品")
            
            # 确定要抓取的页数
            if max_pages is None:
                target_pages = total_pages
                print(f"🎯 将抓取所有 {total_pages} 页")
            else:
                target_pages = min(max_pages, total_pages)
                print(f"🎯 将抓取前 {target_pages} 页（总共{total_pages}页）")
            
            # 逐页抓取商品
            for page_num in range(1, target_pages + 1):
                try:
                    print(f"\n📄 正在处理第 {page_num}/{target_pages} 页...")
                    
                    # 如果不是第一页，需要翻页
                    if page_num > 1:
                        if not self._navigate_to_page(page_num):
                            print(f"❌ 翻页到第 {page_num} 页失败，停止抓取")
                            break
                        
                        # 等待页面加载并验证页面
                        self._human_delay(3, 5)
                        
                        # 验证是否成功翻页
                        new_pagination_info = self._get_pagination_info()
                        if new_pagination_info['current_page'] != page_num:
                            print(f"⚠️  翻页验证失败，期望第{page_num}页，实际第{new_pagination_info['current_page']}页")
                    
                    # 提取当前页商品
                    page_products = self._extract_single_page_products(max_products_per_page)
                    
                    if page_products:
                        # 为每个商品添加页面信息
                        for i, product in enumerate(page_products):
                            product['page_number'] = page_num
                            product['global_index'] = len(products) + i + 1
                            product['page_index'] = i + 1
                        
                        products.extend(page_products)
                        print(f"✅ 第 {page_num} 页获取到 {len(page_products)} 个商品")
                    else:
                        print(f"⚠️  第 {page_num} 页没有找到商品")
                        if page_num == 1:
                            # 第一页没有商品，可能是页面加载问题
                            print("❌ 第一页无商品，可能页面加载异常")
                            break
                        
                    # 添加页面间的延迟（除了最后一页）
                    if page_num < target_pages:
                        self._random_product_browse_delay()
                        
                except Exception as e:
                    self.logger.error(f"处理第 {page_num} 页时出错: {e}")
                    print(f"❌ 第 {page_num} 页处理失败: {e}")
                    continue
            
            print(f"\n📦 JD分页抓取完成:")
            print(f"   ✅ 成功抓取 {len(products)} 个商品")
            print(f"   📄 实际处理 {min(target_pages, len([p for p in products if p.get('page_number')]))} 页")
            
            return products
            
        except Exception as e:
            self.logger.error(f"获取商品列表失败: {e}")
            print(f"❌ 商品列表获取失败: {e}")
            return products

    def _extract_single_page_products(self, max_products: int = None) -> List[Dict[str, Any]]:
        """提取单页商品信息"""
        products = []
        
        try:
            # 使用配置中的选择器
            product_elements = self.driver.find_elements(By.CSS_SELECTOR, JDConfig.SELECTORS['product_list'])
            
            if not product_elements:
                # 尝试备用选择器
                backup_selectors = [
                    '.j-sku-item',
                    '.sku-item',
                    '.product-item',
                    '[data-sku]'
                ]
                
                for selector in backup_selectors:
                    product_elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                    if product_elements:
                        self.logger.info(f"使用备用选择器找到商品: {selector}")
                        break
            
            if max_products:
                product_elements = product_elements[:max_products]
            
            for index, element in enumerate(product_elements):
                try:
                    product_info = self._extract_single_product_info(element, index)
                    if product_info:
                        products.append(product_info)
                        print(f"✅ 提取商品 {len(products)}: {product_info.get('title', '')[:60]}...")
                        print(f"   商品ID: {product_info.get('product_id', '未知')}")
                        print(f"   URL: {product_info.get('url', '未知')}")
                except Exception as e:
                    self.logger.warning(f"提取第{index+1}个商品失败: {e}")
                    continue
            
        except Exception as e:
            self.logger.error(f"提取单页商品失败: {e}")
        
        return products

    def _extract_single_product_info(self, element, index: int) -> Dict[str, Any]:
        """提取单个商品信息"""
        product_info = {
            'index': index,
            'title': '',
            'price': '',
            'product_id': '',
            'url': '',
            'image_url': '',
            'extracted_at': datetime.now().isoformat()
        }
        
        try:
            # 提取商品标题
            title_selectors = ['a[title]', '.p-name a', '.sku-name', '.product-title', 'a']
            for selector in title_selectors:
                try:
                    title_elem = element.find_element(By.CSS_SELECTOR, selector)
                    title = title_elem.get_attribute('title') or title_elem.text.strip()
                    if title:
                        product_info['title'] = title
                        break
                except:
                    continue
            
            # 提取商品链接和ID
            link_selectors = ['a[href*="item.jd.com"]', 'a[href*="/item/"]', 'a']
            for selector in link_selectors:
                try:
                    link_elem = element.find_element(By.CSS_SELECTOR, selector)
                    href = link_elem.get_attribute('href')
                    if href and ('item.jd.com' in href or '/item/' in href):
                        product_info['url'] = href
                        
                        # 从URL提取商品ID
                        match = re.search(r'/(\d+)\.html', href)
                        if match:
                            product_info['product_id'] = match.group(1)
                        break
                except:
                    continue
            
            # 如果没有从URL获取到ID，尝试从元素属性获取
            if not product_info['product_id']:
                try:
                    product_id = element.get_attribute('data-sku') or element.get_attribute('data-pid')
                    if product_id:
                        product_info['product_id'] = product_id
                        if not product_info['url']:
                            product_info['url'] = f"https://item.jd.com/{product_id}.html"
                except:
                    pass
            
            # 提取价格
            price_selectors = ['.p-price .price', '.price', '.p-price i', '.current-price']
            for selector in price_selectors:
                try:
                    price_elem = element.find_element(By.CSS_SELECTOR, selector)
                    price_text = price_elem.text.strip()
                    if price_text:
                        product_info['price'] = price_text
                        break
                except:
                    continue
            
            # 提取图片
            img_selectors = ['img', '.p-img img', '.product-img img']
            for selector in img_selectors:
                try:
                    img_elem = element.find_element(By.CSS_SELECTOR, selector)
                    img_src = img_elem.get_attribute('src') or img_elem.get_attribute('data-lazy-img')
                    if img_src:
                        product_info['image_url'] = img_src
                        break
                except:
                    continue
            
            return product_info if product_info['product_id'] else None
            
        except Exception as e:
            self.logger.warning(f"提取商品信息失败: {e}")
            return None

    def _get_pagination_info(self) -> Dict[str, int]:
        """获取分页信息"""
        try:
            pagination_info = {
                'current_page': 1,
                'total_pages': 1,
                'total_products': 0
            }
            
            # 获取总商品数
            try:
                total_element = self.driver.find_element(By.CSS_SELECTOR, '#J_resCount')
                total_text = total_element.text.strip()
                pagination_info['total_products'] = int(total_text) if total_text.isdigit() else 0
            except:
                # 备用选择器
                try:
                    total_element = self.driver.find_element(By.CSS_SELECTOR, '.f-result-sum span')
                    total_text = total_element.text.strip()
                    pagination_info['total_products'] = int(total_text) if total_text.isdigit() else 0
                except:
                    self.logger.warning("无法获取总商品数")
            
            # 获取当前页和总页数信息
            try:
                # 查找分页信息: <b>1</b><em>/</em><i>3</i>
                page_info_selectors = [
                    '#J_topPage .fp-text',
                    '.g-pager .fp-text',
                    '.pagination .page-info'
                ]
                
                for selector in page_info_selectors:
                    try:
                        page_info = self.driver.find_element(By.CSS_SELECTOR, selector)
                        # 提取当前页和总页数
                        current_elem = page_info.find_element(By.TAG_NAME, 'b')
                        total_elem = page_info.find_element(By.TAG_NAME, 'i')
                        
                        pagination_info['current_page'] = int(current_elem.text.strip())
                        pagination_info['total_pages'] = int(total_elem.text.strip())
                        break
                    except:
                        continue
                        
            except Exception as e:
                self.logger.warning(f"获取分页信息失败: {e}")
            
            return pagination_info
            
        except Exception as e:
            self.logger.error(f"分页信息获取失败: {e}")
            return {'current_page': 1, 'total_pages': 1, 'total_products': 0}
    
    def _navigate_to_page(self, page_num: int) -> bool:
        """导航到指定页面 - 智能分页策略"""
        try:
            # 🔧 修复：在导航前检查窗口状态
            try:
                self._validate_and_fix_window_state()
            except Exception as e:
                self.logger.error(f"导航前窗口状态检查失败: {e}")
                return False
            
            # 获取当前分页状态
            current_info = self._get_pagination_info()
            current_page = current_info['current_page']
            
            # 如果已经在目标页面，无需翻页
            if current_page == page_num:
                self.logger.info(f"已在第{page_num}页，无需翻页")
                return True
            
            self.logger.info(f"从第{current_page}页导航到第{page_num}页")
            
            # 策略1: 如果是下一页，优先使用下一页按钮
            if page_num == current_page + 1:
                self.logger.info("使用下一页按钮翻页")
                return self._go_to_next_page()
            
            # 策略2: 尝试直接点击可见的页码链接
            if self._click_direct_page_link(page_num):
                return True
            
            # 策略3: 连续点击下一页到达目标页面
            if page_num > current_page:
                return self._navigate_by_next_pages(current_page, page_num)
            
            # 策略4: 如果目标页小于当前页，暂不支持向前翻页
            self.logger.warning(f"不支持从第{current_page}页向前翻到第{page_num}页")
            return False
            
        except Exception as e:
            self.logger.error(f"导航到第{page_num}页失败: {e}")
            return False
    
    def _click_direct_page_link(self, page_num: int) -> bool:
        """尝试直接点击页码链接"""
        try:
            # 查找显示页码数字的链接
            page_link_strategies = [
                # 策略1: 查找显示具体页码的链接元素
                f'//div[@class="jPage"]//a[text()="{page_num}"]',
                f'//div[contains(@class, "pagination")]//a[text()="{page_num}"]',
                f'//a[text()="{page_num}" and contains(@onclick, "pageGoodsList")]',
                
                # 策略2: 查找onclick属性中包含目标页码的元素
                f'//a[contains(@onclick, "pageGoodsList({page_num},")]',
                
                # 策略3: 通用分页链接查找
                f'//a[contains(@class, "page-link") and text()="{page_num}"]'
            ]
            
            for xpath in page_link_strategies:
                try:
                    page_links = self.driver.find_elements(By.XPATH, xpath)
                    for link in page_links:
                        if link.is_displayed() and link.is_enabled():
                            # 检查是否是当前页（通常当前页不可点击或有特殊样式）
                            class_name = link.get_attribute('class') or ''
                            if 'current' not in class_name.lower() and 'active' not in class_name.lower():
                                self.logger.info(f"找到页码{page_num}的直接链接")
                                
                                # 使用人性化点击
                                if self._simulate_human_behavior(link):
                                    self.logger.info(f"成功人性化点击页码 {page_num}")
                                    return True
                                else:
                                    # 备用JavaScript点击
                                    self.driver.execute_script("arguments[0].click();", link)
                                    self.logger.info(f"备用JavaScript点击页码 {page_num}")
                                    return True
                except Exception as e:
                    self.logger.debug(f"页码链接策略失败: {xpath}, 错误: {e}")
                    continue
            
            self.logger.info(f"未找到页码{page_num}的直接链接")
            return False
            
        except Exception as e:
            self.logger.error(f"直接点击页码链接失败: {e}")
            return False
    
    def _navigate_by_next_pages(self, current_page: int, target_page: int) -> bool:
        """通过连续点击下一页到达目标页面"""
        try:
            clicks_needed = target_page - current_page
            self.logger.info(f"需要连续点击{clicks_needed}次下一页")
            
            for i in range(clicks_needed):
                current_attempt = current_page + i + 1
                self.logger.info(f"点击下一页到第{current_attempt}页 ({i+1}/{clicks_needed})")
                
                if not self._go_to_next_page():
                    self.logger.error(f"点击下一页失败，停在第{current_page + i}页")
                    return False
                
                # 等待页面加载
                self._human_delay(2, 4)
                
                # 验证是否成功翻页
                new_info = self._get_pagination_info()
                actual_page = new_info['current_page']
                
                if actual_page != current_attempt:
                    self.logger.warning(f"翻页验证失败: 期望第{current_attempt}页，实际第{actual_page}页")
                    # 如果实际页面比期望页面大，说明翻页过快，继续验证
                    if actual_page >= target_page:
                        self.logger.info(f"已到达或超过目标页面: 第{actual_page}页")
                        return True
                else:
                    self.logger.info(f"✅ 成功到达第{actual_page}页")
            
            # 最终验证
            final_info = self._get_pagination_info()
            if final_info['current_page'] == target_page:
                self.logger.info(f"✅ 成功导航到目标页面: 第{target_page}页")
                return True
            else:
                self.logger.error(f"❌ 导航失败: 目标第{target_page}页，实际第{final_info['current_page']}页")
                return False
                
        except Exception as e:
            self.logger.error(f"连续翻页导航失败: {e}")
            return False
    
    def _go_to_next_page(self) -> bool:
        """翻到下一页 - 智能识别下一页按钮"""
        try:
            # 🔧 修复：在操作前检查窗口状态
            try:
                self._validate_and_fix_window_state()
            except Exception as e:
                self.logger.error(f"翻页前窗口状态检查失败: {e}")
                return False
            
            # 获取当前页码，用于构建动态选择器
            current_info = self._get_pagination_info()
            current_page = current_info['current_page']
            next_page = current_page + 1
            
            # 下一页按钮查找策略（按优先级排序）
            next_button_strategies = [
                # 策略1: 动态onclick下一页按钮（最准确）
                f'//a[contains(@onclick, "pageGoodsList({current_page} + 1")]',
                f'//a[contains(@onclick, "pageGoodsList({next_page},")]',
                
                # 策略2: 顶部分页区域的下一页按钮
                '//div[@id="J_topPage"]//a[contains(@class, "fp-next") and not(contains(@class, "disabled"))]',
                '//div[contains(@class, "g-pager")]//a[contains(@class, "fp-next") and not(contains(@class, "disabled"))]',
                
                # 策略3: 底部分页区域的下一页按钮
                '//div[@class="jPage"]//a[contains(text(), "下一页")]',
                '//div[@class="jPage"]//a[text()=">"]',
                '//div[@class="jPage"]//a[contains(@onclick, "pageGoodsList") and contains(text(), ">")]',
                
                # 策略4: 通用下一页选择器
                '//a[contains(text(), "下一页") and not(contains(@class, "disabled"))]',
                '//a[@aria-label="下一页"]',
                '//a[contains(@class, "next") and not(contains(@class, "disabled"))]'
            ]
            
            for i, xpath in enumerate(next_button_strategies):
                try:
                    next_buttons = self.driver.find_elements(By.XPATH, xpath)
                    
                    for btn in next_buttons:
                        if btn.is_displayed() and btn.is_enabled():
                            # 额外检查：确保不是disabled状态
                            class_name = btn.get_attribute('class') or ''
                            onclick_attr = btn.get_attribute('onclick') or ''
                            
                            if 'disabled' not in class_name.lower():
                                self.logger.info(f"使用策略{i+1}找到下一页按钮")
                                self.logger.debug(f"按钮信息: class='{class_name}', onclick='{onclick_attr}'")
                                
                                # 使用人性化点击
                                if self._simulate_human_behavior(btn):
                                    self.logger.info("✅ 成功使用人性化方式点击下一页")
                                    self._human_delay(3, 5)
                                    return True
                                else:
                                    # 备用JavaScript点击
                                    self.driver.execute_script("arguments[0].click();", btn)
                                    self.logger.info("✅ 成功使用JavaScript点击下一页")
                                    self._human_delay(3, 5)
                                    return True
                                    
                except Exception as e:
                    self.logger.debug(f"下一页策略{i+1}失败: {e}")
                    continue
            
            # 如果所有策略都失败，尝试最后的备用方案
            self.logger.warning("常规策略失败，尝试备用方案...")
            return self._try_fallback_next_page()
            
        except Exception as e:
            self.logger.error(f"翻页过程出错: {e}")
            return False
    
    def _try_fallback_next_page(self) -> bool:
        """备用的下一页点击方案"""
        try:
            # 备用方案1: 查找所有可能的下一页元素
            fallback_selectors = [
                'a[onclick*="pageGoodsList"]',
                '.fp-next',
                '.page-next',
                '.next'
            ]
            
            for selector in fallback_selectors:
                try:
                    elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                    for elem in elements:
                        if elem.is_displayed() and elem.is_enabled():
                            text = elem.text.strip()
                            onclick = elem.get_attribute('onclick') or ''
                            
                            # 检查是否像下一页按钮
                            if any(indicator in text.lower() for indicator in ['下一页', '>', 'next']) or \
                               'pageGoodsList' in onclick:
                                
                                self.logger.info(f"备用方案找到可能的下一页按钮: {text}")
                                self.driver.execute_script("arguments[0].click();", elem)
                                self._human_delay(3, 5)
                                return True
                except:
                    continue
            
            self.logger.error("❌ 所有下一页策略都失败")
            return False
            
        except Exception as e:
            self.logger.error(f"备用翻页方案失败: {e}")
            return False

    def get_product_details(self, products: List[Dict[str, Any]], max_products: int = None) -> List[Dict[str, Any]]:
        """获取商品详情 - 使用点击跳转方式"""
        detailed_products = []
        
        if max_products:
            products = products[:max_products]
        
        try:
            # 🔧 修复：确保从第1页开始处理详情
            print("\n🔄 准备开始详情获取，确保回到第1页...")
            if not self._navigate_to_page(1):
                print("❌ 无法导航到第1页，尝试继续...")
            else:
                print("✅ 已回到第1页")
                self._human_delay(2, 3)
            
            # 按页面分组处理商品
            products_by_page = {}
            for product in products:
                page_num = product.get('page_number', 1)
                if page_num not in products_by_page:
                    products_by_page[page_num] = []
                products_by_page[page_num].append(product)
            
            print(f"📊 商品分页统计:")
            for page_num in sorted(products_by_page.keys()):
                count = len(products_by_page[page_num])
                print(f"   第{page_num}页: {count}个商品")
            
            # 按页面顺序处理
            for page_num in sorted(products_by_page.keys()):
                page_products = products_by_page[page_num]
                print(f"\n📄 开始处理第{page_num}页的{len(page_products)}个商品...")
                
                # 确保在正确的页面
                current_info = self._get_pagination_info()
                if current_info['current_page'] != page_num:
                    print(f"🔄 导航到第{page_num}页...")
                    if not self._navigate_to_page(page_num):
                        print(f"❌ 无法导航到第{page_num}页，跳过该页商品")
                        continue
                    self._human_delay(2, 3)
                
                # 处理当前页的商品
                for product in page_products:
                    i = len(detailed_products)  # 全局计数器
                    print(f"\n🔄 处理商品 {i+1}/{len(products)}: {product.get('title', '')[:60]}...")
                    print(f"   📍 位置: 第{page_num}页第{product.get('page_index', '?')}个商品")
                    
                    # 添加商品间的自然浏览延迟（除了第一个商品）
                    if i > 0:
                        self._random_product_browse_delay()
                    
                    # 处理单个商品详情
                    success = self._process_single_product_detail(product, i + 1, len(products))
                    if success and success != "skip":
                        detailed_products.append(success)
            
            return detailed_products
            
        except Exception as e:
            self.logger.error(f"获取商品详情失败: {e}")
            return detailed_products
    
    def _process_single_product_detail(self, product: Dict[str, Any], current_index: int, total_count: int):
        """处理单个商品详情获取，包含重试逻辑"""
        try:
            # 尝试获取商品详情，支持验证码重试
            max_retries = 2  # 最多重试2次
            retry_count = 0
            
            while retry_count <= max_retries:
                try:
                    if retry_count > 0:
                        print(f"🔄 第{retry_count}次重试获取商品详情...")
                        self._human_delay(2, 4)  # 重试前等待
                        
                        # 重试前确保在正确页面
                        expected_page = product.get('page_number', 1)
                        current_info = self._get_pagination_info()
                        if current_info['current_page'] != expected_page:
                            print(f"🔄 重试前导航到第{expected_page}页...")
                            if not self._navigate_to_page(expected_page):
                                print(f"❌ 无法导航到第{expected_page}页")
                                return None
                            self._human_delay(2, 3)
                    
                    # 使用点击跳转方式
                    click_result = self._click_to_product_detail(product)
                    
                    if click_result == "retry":
                        # 验证码处理后需要重试
                        print("🔄 验证码处理完成，准备重新获取详情...")
                        retry_count += 1
                        continue
                    elif click_result == True:
                        # 成功跳转到详情页，提取详情信息
                        detail_info = self._extract_product_detail()
                        if detail_info:
                            detailed_product = {**product, **detail_info}
                            print(f"✅ 商品 {current_index} 详情获取成功")
                            
                            # 返回商品列表
                            try:
                                self._close_product_detail_tab()
                            except Exception as close_error:
                                self.logger.warning(f"关闭详情页失败: {close_error}")
                                # 确保回到正确页面
                                expected_page = product.get('page_number', 1)
                                self._navigate_to_page(expected_page)
                            
                            return detailed_product
                        else:
                            print(f"❌ 商品 {current_index} 详情提取失败")
                            try:
                                self._close_product_detail_tab()
                            except:
                                pass
                            return None
                    else:
                        # 点击跳转失败或用户选择跳过
                        print(f"❌ 商品 {current_index} 点击跳转失败或被跳过")
                        return "skip"
                        
                except Exception as retry_error:
                    self.logger.error(f"重试过程中出错: {retry_error}")
                    retry_count += 1
                    if retry_count <= max_retries:
                        print(f"⚠️ 重试失败，准备第{retry_count}次重试...")
                        self._human_delay(3, 5)
                        # 确保回到正确页面
                        expected_page = product.get('page_number', 1)
                        self._navigate_to_page(expected_page)
                    else:
                        print(f"❌ 达到最大重试次数，跳过商品 {current_index}")
                        return None
            
            print(f"❌ 商品 {current_index} 最终获取失败")
            return None
            
        except Exception as e:
            self.logger.error(f"处理商品详情时出错: {e}")
            return None

    def _click_to_product_detail(self, product: Dict[str, Any]) -> bool:
        """点击跳转到商品详情页 - 人性化版本"""
        try:
            product_id = product.get('product_id')
            page_index = product.get('page_index', 1) - 1  # 转换为0-based索引
            page_number = product.get('page_number', 1)
            
            print(f"🔄 寻找商品元素 (第{page_number}页第{page_index+1}个, ID: {product_id})...")
            
            # 页面交互预热
            self._page_interaction_warmup()
            
            # 重新获取当前页面的商品元素
            product_elements = self.driver.find_elements(By.CSS_SELECTOR, JDConfig.SELECTORS['product_list'])
            
            print(f"📊 当前页面找到 {len(product_elements)} 个商品元素")
            
            if page_index < len(product_elements):
                element = product_elements[page_index]
                print(f"✅ 找到目标商品元素 (页面内索引: {page_index})")
                
                # 尝试多种点击方式 - 优先使用人性化点击
                click_methods = [
                    ('图片', 'img', True),  # 第三个参数表示是否使用人性化点击
                    ('标题链接', 'a[title], .p-name a, .sku-name a', True),
                    ('整个区域', '', True),  # 点击整个商品区域
                    ('图片备用', 'img', False),  # JavaScript备用
                    ('标题备用', 'a[title], .p-name a, .sku-name a', False),
                ]
                
                for method_name, selector, use_human_click in click_methods:
                    try:
                        print(f"🖱️  尝试{'人性化' if use_human_click else 'JavaScript'}点击商品{method_name}...")
                        
                        if selector:
                            click_elem = element.find_element(By.CSS_SELECTOR, selector)
                        else:
                            click_elem = element
                        
                        # 确保元素可见
                        self.driver.execute_script("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", click_elem)
                        self._human_delay(0.8, 1.5)  # 增加等待时间
                        
                        # 记录点击前的窗口句柄
                        original_handles = self.driver.window_handles
                        
                        # 选择点击方式
                        click_success = False
                        if use_human_click:
                            # 使用人性化点击
                            if self._simulate_human_behavior(click_elem):
                                click_success = True
                        else:
                            # 使用JavaScript点击（备用）
                            self.driver.execute_script("arguments[0].click();", click_elem)
                            click_success = True
                        
                        if not click_success:
                            continue
                        
                        # 等待新窗口/标签页打开 - 使用更长延迟
                        self._human_delay(3, 5)
                        
                        # 检查是否有新窗口打开
                        new_handles = self.driver.window_handles
                        if len(new_handles) > len(original_handles):
                            # 切换到新窗口
                            for handle in new_handles:
                                if handle not in original_handles:
                                    self.driver.switch_to.window(handle)
                                    break
                            
                            # 🔧 修复：立即检查新打开的页面，无论是什么页面
                            current_url = self.driver.current_url
                            print(f"🔍 检查新打开的页面: {current_url}")
                            
                            # 首先检查是否是验证码页面（优先级最高）
                            if self._check_for_verification_issues():
                                print(f"🚨 检测到验证码页面，立即暂停处理...")
                                # 返回验证处理结果：True表示需要重新获取详情，False表示跳过
                                should_retry = self._handle_verification_pause(product)
                                if should_retry:
                                    # 🔧 修复：验证完成后，安全地关闭当前页面并确保窗口状态正确
                                    try:
                                        self._close_current_detail_and_return()
                                        # 确保窗口状态正确
                                        self._ensure_back_to_product_list()
                                        print("✅ 验证完成，窗口状态已恢复，准备重试")
                                    except Exception as e:
                                        print(f"⚠️ 窗口状态恢复时出错: {e}")
                                        # 即使出错也继续重试，让上层处理
                                    return "retry"  # 特殊返回值表示需要重试
                                else:
                                    # 用户选择跳过
                                    try:
                                        self._close_current_detail_and_return()
                                        self._ensure_back_to_product_list()
                                    except Exception as e:
                                        print(f"⚠️ 跳过时窗口状态恢复出错: {e}")
                                    return False
                            
                            # 然后检查是否是正确的商品详情页
                            if 'item.jd.com' in current_url or product_id in current_url:
                                print(f"✅ 通过{method_name}点击成功打开详情页")
                                print(f"✅ 成功跳转到详情页: {current_url}")
                                return True
                            else:
                                # 打开了其他页面，记录并继续尝试其他点击方式
                                print(f"⚠️ 打开了非预期页面: {current_url}")
                                print(f"📋 页面标题: {self.driver.title}")
                                # 关闭这个页面，回到商品列表页，继续尝试其他点击方式
                                self._close_current_detail_and_return()
                                continue
                        
                        # 如果没有新窗口，检查当前页面是否变化
                        current_url = self.driver.current_url
                        print(f"🔍 检查当前页面变化: {current_url}")
                        
                        # 🔧 修复：同样需要先检查验证码，再检查是否是详情页
                        if self._check_for_verification_issues():
                            print(f"🚨 当前页面检测到验证码，立即暂停处理...")
                            should_retry = self._handle_verification_pause(product)
                            if should_retry:
                                # 验证码处理完成，确保窗口状态正确
                                try:
                                    self._ensure_back_to_product_list()
                                    print("✅ 验证完成，窗口状态已恢复，准备重试")
                                except Exception as e:
                                    print(f"⚠️ 窗口状态恢复时出错: {e}")
                                return "retry"
                            else:
                                # 用户选择跳过，确保窗口状态正确
                                try:
                                    self._ensure_back_to_product_list()
                                except Exception as e:
                                    print(f"⚠️ 跳过时窗口状态恢复出错: {e}")
                                return False
                        
                        if 'item.jd.com' in current_url and product_id in current_url:
                            print(f"✅ 通过{method_name}点击成功跳转到详情页")
                            return True
                        elif current_url != self.driver.current_url:  # 页面确实变化了，但不是详情页
                            print(f"⚠️ 页面跳转到非预期地址: {current_url}")
                            # 尝试返回商品列表页继续其他点击方式
                            try:
                                self.driver.back()
                                self._human_delay(2, 3)
                            except:
                                pass
                            continue
                            
                    except Exception as e:
                        self.logger.debug(f"点击{method_name}失败: {e}")
                        continue
                
                # 如果点击失败，尝试直接导航
                product_url = product.get('url')
                if product_url:
                    print("🔄 尝试直接导航到商品详情页...")
                    try:
                        self.driver.execute_script(f"window.open('{product_url}', '_blank');")
                        self._human_delay(2, 3)
                        
                        # 切换到新窗口
                        self.driver.switch_to.window(self.driver.window_handles[-1])
                        current_url = self.driver.current_url
                        
                        if 'item.jd.com' in current_url:
                            print("✅ 直接导航成功打开详情页")
                            return True
                    except Exception as e:
                        self.logger.error(f"直接导航失败: {e}")
            else:
                print(f"❌ 商品索引 {page_index} 超出范围，当前页面只有 {len(product_elements)} 个商品")
                print(f"⚠️  可能页面状态不正确，期望在第{page_number}页")
                return False
            
            return False
            
        except Exception as e:
            self.logger.error(f"点击跳转失败: {e}")
            return False

    def _navigate_to_product_detail_safely(self, product_url: str) -> bool:
        """安全导航到商品详情页（备用方法）"""
        try:
            # 在新标签页中打开
            self.driver.execute_script(f"window.open('{product_url}', '_blank');")
            self._human_delay(2, 3)
            
            # 切换到新标签页
            self.driver.switch_to.window(self.driver.window_handles[-1])
            
            # 验证页面加载
            self.wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            
            current_url = self.driver.current_url
            if 'item.jd.com' in current_url:
                return True
            
            return False
            
        except Exception as e:
            self.logger.error(f"安全导航失败: {e}")
            return False

    def _close_product_detail_tab(self):
        """关闭商品详情标签页并返回商品列表"""
        try:
            # 检查当前窗口数量
            handles = self.driver.window_handles
            current_handle = self.driver.current_window_handle
            
            print(f"🔄 当前有 {len(handles)} 个标签页，关闭当前标签页...")
            
            if len(handles) > 1:
                # 关闭当前标签页
                self.driver.close()
                
                # 切换回原来的窗口（通常是第一个）
                remaining_handles = self.driver.window_handles
                if remaining_handles:
                    self.driver.switch_to.window(remaining_handles[0])
                    print("🔙 已返回商品列表页面")
                else:
                    self.logger.warning("没有可切换的窗口")
            else:
                # 如果只有一个窗口，直接返回
                self.driver.back()
                self._human_delay(1, 2)
                print("🔙 已返回商品列表页面")
        
        except Exception as e:
            self.logger.warning(f"关闭详情页失败: {e}")
            # 尝试恢复
            try:
                handles = self.driver.window_handles
                if handles:
                    self.driver.switch_to.window(handles[0])
            except:
                pass

    def _extract_product_detail(self) -> Dict[str, Any]:
        """提取商品详情信息"""
        detail_info = {
            'product_title': '',
            'price_info': {},
            'basic_info': {},
            'cover_image': '',
            'services': [],
            'shop_info': {},
            'detailed_specifications': {},
            'extracted_at': datetime.now().isoformat()
        }
        
        try:
            # 等待页面加载
            self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, ".w, .product-intro, body")))
            self._human_delay(2, 3)
            
            # 提取各部分信息
            detail_info['product_title'] = self._extract_product_title()
            detail_info['price_info'] = self._extract_price_info()
            detail_info['basic_info'] = self._extract_basic_info()
            detail_info['cover_image'] = self._extract_cover_image()
            detail_info['services'] = self._extract_service_commitments()
            detail_info['shop_info'] = self._extract_shop_info_detail()
            detail_info['detailed_specifications'] = self._extract_detailed_specifications()
            
            self.logger.info("成功提取JD商品详情信息")
            return detail_info
            
        except Exception as e:
            self.logger.error(f"提取商品详情失败: {e}")
            return detail_info

    def _extract_product_title(self) -> str:
        """提取商品标题"""
        try:
            title_selectors = [
                '.sku-name',
                '.itemInfo-wrap .sku-name',
                '.product-intro .sku-name',
                'h1.sku-name',
                '.p-name',
                '.product-name'
            ]
            
            for selector in title_selectors:
                try:
                    title_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    title = title_elem.text.strip()
                    if title:
                        return title
                except:
                    continue
            
            return ""
            
        except Exception as e:
            self.logger.warning(f"提取商品标题失败: {e}")
            return ""

    def _extract_price_info(self) -> Dict[str, str]:
        """提取价格信息"""
        price_info = {}
        
        try:
            # JD价格
            jd_price_selectors = [
                '.price.J-p-', 
                '.p-price .price',
                '.summary-price .price',
                '#jd-price .price',
                '.price'
            ]
            
            for selector in jd_price_selectors:
                try:
                    price_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    price_text = price_elem.text.strip().replace('￥', '').replace(',', '')
                    if price_text and price_text.replace('.', '').isdigit():
                        price_info['jd_price'] = price_text
                        break
                except:
                    continue
            
            # 到手价
            final_price_selectors = [
                '.final-price',
                '.dt-price',
                '.summary-price span',
                '.p-price'
            ]
            
            for selector in final_price_selectors:
                try:
                    price_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    price_text = price_elem.text.strip()
                    if '￥' in price_text:
                        price_info['final_price'] = price_text
                        break
                except:
                    continue
            
            # 如果没有到手价，使用JD价格
            if 'final_price' not in price_info and 'jd_price' in price_info:
                price_info['final_price'] = f"￥{price_info['jd_price']}"
            
            # 原价
            original_price_selectors = [
                '.original-price',
                '.del-price', 
                '.strike-price',
                '.p-price .del'
            ]
            
            for selector in original_price_selectors:
                try:
                    price_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    price_text = price_elem.text.strip()
                    if price_text:
                        price_info['original_price'] = price_text
                        break
                except:
                    continue
            
            self.logger.info(f"提取价格信息: {price_info}")
            return price_info
            
        except Exception as e:
            self.logger.warning(f"提取价格信息失败: {e}")
            return price_info

    def _extract_basic_info(self) -> Dict[str, str]:
        """提取商品基本信息 - 优化版本"""
        basic_info = {}

        try:
            # 精准定位商品详情区域
            goods_base_selector = ".goods-base"
            
            # 等待商品详情区域加载
            try:
                # 增加等待时间，确保页面完全加载
                self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, goods_base_selector)))
                goods_base = self.driver.find_element(By.CSS_SELECTOR, goods_base_selector)
                
                # 额外等待一下，确保动态内容加载完成
                import time
                time.sleep(2)
                
            except Exception as e:
                self.logger.warning(f"未找到商品详情区域: {e}")
                print(f"⚠️  商品详情区域加载失败，当前URL: {self.driver.current_url}")
                return basic_info

            # 提取所有详情项目
            detail_items = goods_base.find_elements(By.CSS_SELECTOR, ".item")
            
            for item in detail_items:
                try:
                    # 获取属性名称
                    name_elem = item.find_element(By.CSS_SELECTOR, ".name")
                    name = name_elem.text.strip()
                    
                    # 获取属性值
                    text_elem = item.find_element(By.CSS_SELECTOR, ".text")
                    value = text_elem.text.strip()
                    
                    if name and value:
                        # 映射关键字段
                        if name == "品牌":
                            basic_info['brand'] = value
                        elif name == "商品编号":
                            basic_info['product_code'] = value
                        elif name == "店铺":
                            basic_info['shop'] = value
                        elif name == "型号":
                            basic_info['model'] = value
                        # 保存所有详情信息供后续使用
                        basic_info[name] = value
                        
                except Exception as e:
                    continue

            # 从URL提取商品ID
            try:
                current_url = self.driver.current_url
                product_id = self._extract_product_id(current_url)
                if product_id:
                    basic_info['product_id'] = product_id
            except:
                pass

            # 提取商品类别（从面包屑导航中获取）
            try:
                category = self._extract_product_category()
                if category:
                    basic_info['category'] = category
                    self.logger.info(f"成功提取商品类别: {category}")
            except Exception as e:
                self.logger.warning(f"提取商品类别失败: {e}")

            self.logger.info(f"成功提取 {len(basic_info)} 个基本信息字段")
            
            # 调试信息：打印提取到的字段
            if self.debug and basic_info:
                print(f"🔍 提取到的商品规格字段:")
                for key, value in basic_info.items():
                    print(f"   {key}: {value}")
            
            return basic_info

        except Exception as e:
            self.logger.warning(f"提取基本信息失败: {e}")
            return basic_info

    def _extract_product_category(self) -> str:
        """提取商品类别（从页面导航路径中获取第三级分类）"""
        try:
            # 查找导航路径容器
            breadcrumb_selectors = [
                '.crumb.clearfix',  # 主要选择器
                '#crumb-wrap .crumb',  # 备用选择器
                '.crumb-wrap .crumb'  # 另一个备用选择器
            ]
            
            breadcrumb_container = None
            for selector in breadcrumb_selectors:
                try:
                    breadcrumb_container = self.driver.find_element(By.CSS_SELECTOR, selector)
                    self.logger.info(f"找到导航路径容器: {selector}")
                    break
                except:
                    continue
            
            if not breadcrumb_container:
                self.logger.warning("未找到导航路径容器")
                return ""
            
            # 查找所有导航项目（排除分隔符）
            breadcrumb_items = breadcrumb_container.find_elements(By.CSS_SELECTOR, '.item:not(.sep)')
            
            if len(breadcrumb_items) < 3:
                self.logger.warning(f"导航路径项目数量不足，只有 {len(breadcrumb_items)} 个")
                return ""
            
            # 获取第三个项目（索引为2）- 这通常是商品的主要类别
            third_item = breadcrumb_items[2]
            
            # 尝试从链接或文本中提取类别名称
            category_text = ""
            try:
                # 优先从链接文本获取
                link = third_item.find_element(By.TAG_NAME, 'a')
                category_text = link.text.strip()
            except:
                # 如果没有链接，直接获取文本内容
                category_text = third_item.text.strip()
            
            if category_text:
                self.logger.info(f"成功提取商品类别: {category_text}")
                return category_text
            else:
                self.logger.warning("第三级分类项目为空")
                return ""
                
        except Exception as e:
            self.logger.error(f"提取商品类别异常: {e}")
            return ""

    def _extract_product_id(self, url: str) -> str:
        """从URL提取商品ID"""
        try:
            match = re.search(r'/(\d+)\.html', url)
            return match.group(1) if match else ""
        except:
            return ""

    def _extract_cover_image(self) -> str:
        """提取商品封面图片"""
        try:
            image_selectors = [
                '#spec-img',
                '.jqzoom img',
                '.main-img img',
                '.preview img',
                '.product-img img',
                '.p-img img',
                'img[alt*="商品"]',
                '.itemInfo-wrap img'
            ]
            
            for selector in image_selectors:
                try:
                    img_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    
                    # 检查多个属性
                    img_src = (img_elem.get_attribute('src') or 
                              img_elem.get_attribute('data-origin') or
                              img_elem.get_attribute('data-lazy-img') or
                              img_elem.get_attribute('data-src'))
                    
                    if img_src and 'http' in img_src:
                        self.logger.info(f"成功提取封面图片: {img_src[:50]}...")
                        return img_src
                except:
                    continue
            
            # 如果没找到，尝试通用图片搜索
            try:
                images = self.driver.find_elements(By.TAG_NAME, 'img')
                for img in images:
                    src = img.get_attribute('src')
                    if src and '360buyimg.com' in src and 'n5' in src:
                        self.logger.info(f"备用方式提取封面图片: {src[:50]}...")
                        return src
            except:
                pass
            
            return ""
            
        except Exception as e:
            self.logger.warning(f"提取封面图片失败: {e}")
            return ""

    def _extract_service_commitments(self) -> List[str]:
        """提取服务承诺"""
        services = []
        
        try:
            service_selectors = [
                '.services a',
                '.promise-icon a',
                '.service-support a',
                '.summary-stock .services a'
            ]
            
            for selector in service_selectors:
                try:
                    service_elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                    for elem in service_elements:
                        service_text = elem.text.strip()
                        if service_text and service_text not in services:
                            services.append(service_text)
                except:
                    continue
                    
        except Exception as e:
            self.logger.warning(f"提取服务承诺失败: {e}")
        
        return services

    def _extract_shop_info_detail(self) -> Dict[str, str]:
        """提取店铺详细信息"""
        shop_info = {}
        
        try:
            # 提取店铺名称
            shop_name_selectors = [
                '.shop-name',
                '.store-name', 
                '.detail-shop .name',
                '.shop-info .name'
            ]
            
            for selector in shop_name_selectors:
                try:
                    shop_elem = self.driver.find_element(By.CSS_SELECTOR, selector)
                    shop_info['name'] = shop_elem.text.strip()
                    break
                except:
                    continue
        
        except Exception as e:
            self.logger.warning(f"提取店铺信息失败: {e}")

        return shop_info

    def _extract_detailed_specifications(self) -> Dict[str, str]:
        """提取详细规格参数 - 已整合到basic_info中"""
        # 所有规格参数已经在_extract_basic_info中完整提取
        # 此方法保留为兼容性占位符，避免无用日志信息
        return {}

    def convert_to_api_format(self, product: Dict[str, Any]) -> Dict[str, Any]:
        """转换为API所需格式"""
        try:
            # 基础信息
            basic_info = product.get('basic_info', {})
            price_info = product.get('price_info', {})
            services = product.get('services', [])
            
            # 调试信息
            if self.debug:
                self.logger.debug(f"转换商品: {product.get('product_title', '')[:50]}...")
                self.logger.debug(f"basic_info字段数: {len(basic_info)}")
                self.logger.debug(f"price_info: {price_info}")
                self.logger.debug(f"services数量: {len(services)}")
            
            # 构建product_data字符串
            product_data_parts = []
            
            # 基本信息
            if basic_info.get('brand'):
                product_data_parts.append(f"品牌: {basic_info['brand']}")
            if basic_info.get('model'):
                product_data_parts.append(f"型号: {basic_info['model']}")
            if basic_info.get('product_code'):
                product_data_parts.append(f"商品编号: {basic_info['product_code']}")
            
            # 价格信息
            if price_info.get('jd_price'):
                product_data_parts.append(f"京东价: {price_info['jd_price']}")
            if price_info.get('final_price'):
                product_data_parts.append(f"到手价: {price_info['final_price']}")
            
            # 服务信息
            if services:
                service_text = " · ".join(services)
                product_data_parts.append(f"服务: {service_text}")
            
            # 自适应规格提取 - 包含所有从basic_info中提取到的规格信息
            spec_parts = []
            
            # 定义需要排除的基础字段（这些已经在上面处理过了）
            excluded_fields = {
                'brand', 'product_code', 'shop', 'model', 'product_id',
                '品牌', '商品编号', '店铺', '型号'
            }
            
            # 定义需要过滤的无效值
            invalid_values = {'-', '暂无', '无', '不支持', '其他', '', ' ', '　'}
            
            # 遍历所有basic_info中的字段，自动包含有效的规格信息
            for spec_name, value in basic_info.items():
                # 跳过已处理的基础字段
                if spec_name in excluded_fields:
                    continue
                
                # 确保值有效且不为空
                if value and str(value).strip() not in invalid_values:
                    # 清理值中的多余空白
                    clean_value = str(value).strip()
                    if clean_value:
                        spec_parts.append(f"{spec_name}: {clean_value}")

            # 添加所有有效规格到product_data中
            if spec_parts:
                # 按字段名排序，确保输出一致性
                spec_parts.sort()
                product_data_parts.extend(spec_parts)
                
                # 调试信息
                if len(spec_parts) < 5:
                    self.logger.warning(f"商品规格字段较少，仅 {len(spec_parts)} 个: {spec_parts}")
                else:
                    self.logger.info(f"成功添加 {len(spec_parts)} 个商品规格字段")
            else:
                self.logger.warning(f"未找到任何有效的商品规格字段，basic_info内容: {basic_info}")

            # 构建API格式数据
            api_data = {
                'home_image': product.get('cover_image', '') or product.get('image_url', ''),
                'product_name': product.get('product_title', '') or product.get('title', ''),
                'product_type': basic_info.get('category', '') or basic_info.get('brand', ''),
                'product_data': '; '.join(product_data_parts) if product_data_parts else '',
                'product_url': product.get('url', ''),
                'platform_product_id': product.get('product_id', '') or basic_info.get('product_id', '')
            }
            
            # 调试信息：记录product_type的来源
            if basic_info.get('category'):
                self.logger.info(f"✅ 使用商品类别作为product_type: {basic_info.get('category')}")
            elif basic_info.get('brand'):
                self.logger.info(f"⚠️ 使用品牌名作为product_type（未找到商品类别）: {basic_info.get('brand')}")
            else:
                self.logger.warning("❌ 未找到product_type数据（既无商品类别也无品牌）")
            
            return api_data
            
        except Exception as e:
            self.logger.error(f"转换API格式失败: {e}")
            return {}

    def submit_to_api(self, products: List[Dict[str, Any]], store_id: str) -> bool:
        """提交数据到API"""
        try:
            # 导入API配置
            import sys
            import os
            sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
            from api_config import APIConfig
            
            # 获取API配置
            api_url = APIConfig.get_products_api_url()
            
            if APIConfig.ENABLE_REQUEST_LOGGING:
                # 打印原始提取的商品数据
                print("\n" + "="*80)
                print("📦 JD原始提取的商品数据:")
                print("="*80)
                for i, product in enumerate(products, 1):
                    print(f"\n🔍 原始商品 {i} 数据:")
                    for key, value in product.items():
                        if key == 'element':
                            continue  # 跳过WebElement对象
                        if isinstance(value, str) and len(value) > 100:
                            print(f"  {key}: {value[:100]}...")
                        else:
                            print(f"  {key}: {value}")
                print("="*80)
            
            # 直接使用传入的API格式数据（GUI监听器已经转换过了）
            api_products = products
            
            # 构建请求数据
            payload = {
                "store_id": store_id,
                "platform": APIConfig.PLATFORM_MAPPING.get("京东", "JD"),
                "products": api_products
            }
            
            if APIConfig.ENABLE_REQUEST_LOGGING:
                # 打印详细的API数据到控制台
                print("\n" + "="*80)
                print("🚀 JD准备提交到API的数据详情:")
                print("="*80)
                print(f"📋 Store ID: {store_id}")
                print(f"🏪 Platform: {APIConfig.PLATFORM_MAPPING.get('京东', 'JD')}")
                print(f"📦 商品数量: {len(api_products)}")
                print(f"🌐 API地址: {api_url}")
                print("-"*80)
                
                for i, product in enumerate(api_products, 1):
                    print(f"\n📦 商品 {i}:")
                    print(f"  🏷️  商品名称: {product.get('product_name', 'N/A')}")
                    print(f"  🆔 商品ID: {product.get('platform_product_id', 'N/A')}")
                    # 从product_data字符串中提取价格
                    product_data_str = product.get('product_data', '')
                    price = 'N/A'
                    if isinstance(product_data_str, str) and '京东价:' in product_data_str:
                        try:
                            # 从字符串中提取价格信息
                            import re
                            price_match = re.search(r'京东价:\s*([^;]+)', product_data_str)
                            if price_match:
                                price = price_match.group(1).strip()
                        except:
                            pass
                    print(f"  💰 价格: {price}")
                    print(f"  🖼️  封面图: {product.get('home_image', 'N/A')[:80]}{'...' if len(str(product.get('home_image', ''))) > 80 else ''}")
                    print(f"  🔗 商品链接: {product.get('product_url', 'N/A')}")
                
                print("="*80)
                print("🔄 开始提交到API...")
                print("="*80)

            # 发送请求（支持重试）
            import requests
            import time
            
            for attempt in range(APIConfig.MAX_RETRIES):
                try:
                    if APIConfig.ENABLE_VERBOSE_LOGGING:
                        print(f"🔄 第 {attempt + 1}/{APIConfig.MAX_RETRIES} 次尝试提交...")
                    
                    response = requests.post(
                        api_url,
                        json=payload,
                        headers=APIConfig.DEFAULT_HEADERS,
                        timeout=APIConfig.REQUEST_TIMEOUT
                    )
                    
                    if APIConfig.ENABLE_RESPONSE_LOGGING:
                        print(f"📡 API响应状态码: {response.status_code}")
                        if response.text:
                            print(f"📄 API响应内容: {response.text[:500]}...")
                    
                    if response.status_code == 200:
                        print(f"✅ JD API提交成功: {len(api_products)} 个商品")
                        self.logger.info(f"JD API提交成功: store_id={store_id}, 商品数={len(api_products)}")
                        return True
                    else:
                        error_msg = f"API返回错误状态码: {response.status_code}"
                        if response.text:
                            error_msg += f" - {response.text}"
                        
                        if attempt < APIConfig.MAX_RETRIES - 1:
                            print(f"⚠️ {error_msg}，{APIConfig.RETRY_DELAY}秒后重试...")
                            time.sleep(APIConfig.RETRY_DELAY)
                        else:
                            print(f"❌ JD API提交失败: {error_msg}")
                            self.logger.error(f"JD API提交失败: {error_msg}")
                            return False
                            
                except requests.RequestException as e:
                    error_msg = f"网络请求异常: {e}"
                    if attempt < APIConfig.MAX_RETRIES - 1:
                        print(f"⚠️ {error_msg}，{APIConfig.RETRY_DELAY}秒后重试...")
                        time.sleep(APIConfig.RETRY_DELAY)
                    else:
                        print(f"❌ JD API提交失败: {error_msg}")
                        self.logger.error(f"JD API提交失败: {error_msg}")
                        return False
            
            return False
                
        except Exception as e:
            self.logger.error(f"JD API提交异常: {e}")
            print(f"❌ JD API提交异常: {e}")
            return False

    def print_api_format_data(self, products: List[Dict[str, Any]]):
        """打印API格式数据"""
        try:
            print("\n" + "="*80)
            print("🔄 JD商品数据转换为API格式")
            print("="*80)

            api_products = []

            for index, product in enumerate(products, 1):
                print(f"\n【商品 {index}】")
                print("-" * 50)

                # 转换为API格式
                api_data = self.convert_to_api_format(product)
                api_products.append(api_data)

                # 打印API格式数据
                for key, value in api_data.items():
                    print(f"{key}: {value}")

            # 保存API格式数据到文件
            api_file = "jd_products_api_format.json"
            with open(api_file, 'w', encoding='utf-8') as f:
                json.dump(api_products, f, ensure_ascii=False, indent=2)

            print(f"\n💾 API格式数据已保存到: {api_file}")
            print(f"📊 共转换 {len(api_products)} 个商品")
            print("="*80)

        except Exception as e:
            self.logger.error(f"打印API格式数据失败: {e}")
            print(f"❌ 数据格式化失败: {e}")

    def _display_api_data(self, api_data_list: List[Dict[str, Any]]):
        """显示已转换的API格式数据"""
        try:
            print("\n" + "="*80)
            print("🔄 JD商品数据转换为API格式")
            print("="*80)

            for index, api_data in enumerate(api_data_list, 1):
                print(f"\n【商品 {index}】")
                print("-" * 50)

                # 打印API格式数据
                for key, value in api_data.items():
                    print(f"{key}: {value}")

            print(f"\n💾 API格式数据已保存到: jd_products_api_format.json")
            print(f"📊 共转换 {len(api_data_list)} 个商品")
            print("="*80)

        except Exception as e:
            self.logger.error(f"显示API格式数据失败: {e}")
            print(f"❌ 数据显示失败: {e}")

    def save_data(self, products: List[Dict[str, Any]], formats: List[str] = None) -> Dict[str, str]:
        """保存数据到文件"""
        if formats is None:
            formats = ['json']
        
        saved_files = {}
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        
        try:
            for fmt in formats:
                if fmt.lower() == 'json':
                    filename = f"{JDConfig.OUTPUT_DIR}/jd_products.json"
                    with open(filename, 'w', encoding='utf-8') as f:
                        json.dump(products, f, ensure_ascii=False, indent=2)
                    saved_files['json'] = filename
                    self.logger.info(f"数据已保存为JSON格式: {filename}")
                
                elif fmt.lower() == 'txt':
                    filename = f"{JDConfig.OUTPUT_DIR}/jd_products_formatted.txt"
                    with open(filename, 'w', encoding='utf-8') as f:
                        for i, product in enumerate(products, 1):
                            f.write(f"=== 商品 {i} ===\n")
                            f.write(f"商品标题: {product.get('product_title', product.get('title', ''))}\n")
                            f.write(f"商品ID: {product.get('product_id', '')}\n")
                            f.write(f"URL: {product.get('url', '')}\n")
                            
                            basic_info = product.get('basic_info', {})
                            f.write(f"商品详情字段数: {len(basic_info)}\n")
                            
                            for key, value in basic_info.items():
                                f.write(f"  {key}: {value}\n")
                            f.write("\n")
                    
                    saved_files['txt'] = filename
                    self.logger.info(f"数据已保存为文本格式: {filename}")
            
            return saved_files
            
        except Exception as e:
            self.logger.error(f"保存数据失败: {e}")
            return saved_files

    def __enter__(self):
        """上下文管理器入口"""
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """上下文管理器出口"""
        self.close()

    def close(self):
        """关闭浏览器"""
        if self.driver:
            try:
                self.driver.quit()
                self.logger.info("浏览器已关闭")
            except Exception as e:
                self.logger.warning(f"关闭浏览器时出错: {e}")


def scrape_all_products(shop_url: str, max_pages: int = None, max_products_per_page: int = None):
    """
    爬取店铺所有商品的完整流程
    
    Args:
        shop_url: JD店铺URL
        max_pages: 最大爬取页数，None表示爬取所有页
        max_products_per_page: 每页最大商品数，None表示爬取所有商品
    
    Returns:
        list: API格式的商品数据列表，失败返回False
    """
    print("🏪 JD商城全量商品采集器")
    print("=" * 80)
    print(f"🎯 目标店铺: {shop_url}")
    print(f"📄 页数限制: {'无限制' if max_pages is None else max_pages}")
    print(f"📦 商品限制: {'无限制' if max_products_per_page is None else max_products_per_page}")
    print("-" * 80)
    
    try:
        with JDScraper(headless=False, debug=True) as scraper:
            print("\n🔐 步骤1: JD登录")
            if not scraper.login_jd():
                print("❌ 登录失败，无法继续")
                return False
            
            print("\n🏪 步骤2: 访问店铺")
            if not scraper.navigate_to_shop(shop_url):
                print("❌ 访问店铺失败，无法继续")
                return False
            
            print("\n📋 步骤3: 获取商品列表")
            products = scraper.get_product_list(
                max_pages=max_pages,
                max_products_per_page=max_products_per_page
            )
            
            if not products:
                print("❌ 未获取到商品列表")
                return False
            
            print(f"✅ 获取到 {len(products)} 个商品")
            
            print("\n🔍 步骤4: 提取商品详情")
            detailed_products = scraper.get_product_details(products)
            
            if not detailed_products:
                print("❌ 详情提取失败")
                return False
            
            print(f"✅ 成功提取 {len(detailed_products)} 个商品详情")
            
            print("\n💾 步骤5: 保存数据")
            saved_files = scraper.save_data(detailed_products, ['json', 'txt'])
            
            print(f"✅ 数据已保存:")
            for format_type, file_path in saved_files.items():
                print(f"   📄 {format_type.upper()}: {file_path}")
            
            print("\n🔄 步骤6: 转换API格式")
            api_data_list = []
            for product in detailed_products:
                api_data = scraper.convert_to_api_format(product)
                if api_data:
                    api_data_list.append(api_data)
            
            if api_data_list:
                # 保存API格式数据
                api_file = 'jd_products_api_format.json'
                with open(api_file, 'w', encoding='utf-8') as f:
                    json.dump(api_data_list, f, ensure_ascii=False, indent=2)
                
                print(f"✅ API格式数据已保存: {api_file}")
                print(f"📊 共转换 {len(api_data_list)} 个商品")
                
                # 显示API格式预览
                scraper._display_api_data(api_data_list)
                
                return api_data_list
            else:
                print("❌ API格式转换失败")
                return False
                
    except Exception as e:
        print(f"❌ 爬取过程中出错: {e}")
        import traceback
        traceback.print_exc()
        return False


# 主入口
if __name__ == "__main__":
    print("🎯 JD商城全量商品采集器")
    print("=" * 80)
    print("📋 功能: 自动登录 → 访问店铺 → 获取商品列表 → 提取详情 → 保存数据 → API格式转换")
    print("⚠️  使用点击跳转避免HSTS检测，支持图片、标题、区域多种点击方式")
    print("-" * 80)
    
    # 默认配置
    default_shop_url = "https://mall.jd.com/view_search-2253208-0-99-1-24-1.html"
    
    try:
        print(f"\n🏪 店铺URL (回车使用默认): ")
        print(f"默认: {default_shop_url}")
        shop_url = input("输入店铺URL: ").strip()
        
        if not shop_url:
            shop_url = default_shop_url
            print(f"✅ 使用默认店铺: {shop_url}")
        
        print(f"\n📄 页数限制 (回车表示爬取所有页): ")
        max_pages_input = input("最大页数: ").strip()
        max_pages = int(max_pages_input) if max_pages_input else None
        
        print(f"\n📦 每页商品数限制 (回车表示爬取所有商品): ")
        max_products_input = input("每页最大商品数: ").strip()
        max_products_per_page = int(max_products_input) if max_products_input else None
        
        print(f"\n🚀 开始执行全量商品采集...")
        
        # 执行爬取
        result = scrape_all_products(
            shop_url=shop_url,
            max_pages=max_pages,
            max_products_per_page=max_products_per_page
        )
        
        if result:
            print(f"\n🎉 全量商品采集完成!")
            print(f"✅ 共采集 {len(result)} 个商品的完整信息")
            print(f"✅ 数据已保存为多种格式")
            print(f"✅ API格式数据已准备完毕")
        else:
            print(f"\n❌ 全量商品采集失败")
            
    except KeyboardInterrupt:
        print("\n🛑 用户取消操作")
    except Exception as e:
        print(f"❌ 执行过程中出错: {e}")
        import traceback
        traceback.print_exc()
    
    print("\n🏁 程序执行完成!") 